{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Watson Machine Leaning testing with blind set\n",
    "\n",
    "In this notebook we will score our deployed model using images never used by training or evaluation. Add a zip folder with your blind set of images to you Watson Studio Project.\n",
    "\n",
    "This is the second notebook in the process, please make sure that you have deployed your model using the training notebook before using this one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image as PImage\n",
    "from object_detection.utils import visualization_utils as vis_util\n",
    "from object_detection.utils import label_map_util\n",
    "import numpy as np\n",
    "import zipfile\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import ibm_boto3\n",
    "from ibm_botocore.client import Config, ClientError\n",
    "\n",
    "def download_file_cos(local_file_name, key): \n",
    "    '''\n",
    "    Wrapper function to download a file from cloud object storage using the\n",
    "    credential dict provided and loading it into memory\n",
    "    '''\n",
    "    cos = ibm_boto3.client(\"s3\",\n",
    "        ibm_api_key_id=credentials_cos['IAM_SERVICE_ID'],\n",
    "        ibm_service_instance_id=credentials_cos['IBM_API_KEY_ID'],\n",
    "        config=Config(signature_version=\"oauth\"),\n",
    "        endpoint_url=credentials_cos['ENDPOINT']\n",
    "    )\n",
    "    try:\n",
    "        res=cos.download_file(Bucket=credentials_cos['BUCKET'], Key=key, Filename=local_file_name)\n",
    "    except Exception as e:\n",
    "        print('Exception', e)\n",
    "    else:\n",
    "        print('File Downloaded')\n",
    "        \n",
    "def get_annotations(): \n",
    "    cos = ibm_boto3.client(\"s3\",\n",
    "        ibm_api_key_id=credentials_cos['IBM_API_KEY_ID'],\n",
    "        ibm_service_instance_id=credentials_cos['IAM_SERVICE_ID'],\n",
    "        config=Config(signature_version=\"oauth\"),\n",
    "        endpoint_url=credentials_cos['ENDPOINT']\n",
    "    )\n",
    "    try:\n",
    "        return json.loads(cos.get_object(Bucket=credentials_cos['BUCKET'], Key='_annotations.json')['Body'].read())\n",
    "    except Exception as e:\n",
    "        print('Exception', e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the credentials using Watson Studio\n",
    "\n",
    "In your data tab (101 Symbol) select your zip and using the option \"Insert to Code\" -> \"Credentials\". A cell just like the one below will pop, change the name of the variable to credentials_cos:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# @hidden_cell\n",
    "# The following code contains the credentials for a file in your IBM Cloud Object Storage.\n",
    "# You might want to remove those credentials before you share your notebook.\n",
    "credentials_cos = {\n",
    "    'IAM_SERVICE_ID': '<IAM_SERVICE_ID>',\n",
    "    'IBM_API_KEY_ID': '<IBM_API_KEY_ID>',\n",
    "    'ENDPOINT': '<ENDPOINT>',\n",
    "    'IBM_AUTH_ENDPOINT': '<IBM_AUTH_ENDPOINT>',\n",
    "    'BUCKET': '<BUCKET>',\n",
    "    'FILE': '<FILE>'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pathlib\n",
    "\n",
    "# Clone the tensorflow models repository if it doesn't already exist\n",
    "if \"models\" in pathlib.Path.cwd().parts:\n",
    "    while \"models\" in pathlib.Path.cwd().parts:\n",
    "        os.chdir('..')\n",
    "elif not pathlib.Path('models').exists():\n",
    "    !git clone --depth 1 https://github.com/cloud-annotations/models\n",
    "\n",
    "%cd models/research\n",
    "!protoc object_detection/protos/*.proto --python_out=."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.exists('blind') and os.path.isdir('blind'):\n",
    "    shutil.rmtree('blind')\n",
    "    \n",
    "os.makedirs('blind', exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use your file name or credentials_cos['File'] to add your zip to the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_file_cos(os.path.join('blind', 'guns_test.zip'), 'guns_test.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with zipfile.ZipFile(os.path.join('blind', 'guns_test.zip'), 'r') as zip_ref:\n",
    "    zip_ref.extractall('blind')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%ls blind"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring Step\n",
    "\n",
    "You will need a Watson Machine Leaning instance and an IAM API Key in IBM Cloud that has access to this instance. See the steps in the documentation:\n",
    "\n",
    "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/ml-authentication.html\n",
    "\n",
    "Also, in the new version of WML you will need a Deployment Space and it's ID\n",
    "\n",
    "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/ml-spaces_local.html?audience=wdp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ibm_watson_machine_learning import APIClient\n",
    "\n",
    "wml_credentials = {\n",
    "                   \"url\": \"https://us-south.ml.cloud.ibm.com\",\n",
    "                   \"apikey\":\"<apikey>\"\n",
    "                  }\n",
    "\n",
    "client = APIClient(wml_credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.set.default_space(\"<deployment-space-id>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the deployment ID of your model\n",
    "\n",
    "The next cell will list all the deployments available in your instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.deployments.list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['N/A','Pistol','Rifle']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('blind/label_map.pbtxt', 'w') as f:\n",
    "  # Loop through all of the labels and write each label to the file with an id\n",
    "  for idx, label in enumerate(labels):\n",
    "    f.write('item {\\n')\n",
    "    f.write(\"\\tname: '{}'\\n\".format(label))\n",
    "    f.write('\\tid: {}\\n'.format(idx + 1)) # indexes must start at 1\n",
    "    f.write('}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_index = label_map_util.create_category_index_from_labelmap('blind/label_map.pbtxt', use_display_name=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_images = []\n",
    "for filename in os.listdir('blind'):\n",
    "    if filename.endswith(\"jpeg\"): \n",
    "        test_images.append(filename)\n",
    "        continue\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = PImage.open(os.path.join('blind', test_images[0]))\n",
    "# the array based representation of the image will be used later in order to prepare the\n",
    "# result image with boxes and labels on it.\n",
    "(im_width, im_height) = image.size\n",
    "image_np = np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = image_np.tolist()\n",
    "payload_scoring = {\n",
    "  \"input_data\": [{\n",
    "    \"values\": [data]\n",
    "  }]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use the deployment ID of your model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "predictions = client.deployments.score('<deployment-id>', payload_scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in predictions['predictions']:\n",
    "    if x['id'] == 'detection_scores':\n",
    "        scores = x['values'][0]\n",
    "    if x['id'] == 'detection_boxes':\n",
    "        boxes = x['values'][0]\n",
    "    if x['id'] == 'num_detections':\n",
    "        num = x['values'][0]\n",
    "    if x['id'] == 'detection_classes':\n",
    "        classes = x['values'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "vis_util.visualize_boxes_and_labels_on_image_array(\n",
    "            image_np,\n",
    "            np.squeeze(boxes),\n",
    "            np.squeeze(classes).astype(np.int32),\n",
    "            np.squeeze(scores),\n",
    "            category_index,\n",
    "            use_normalized_coordinates=True,\n",
    "            line_thickness=8)\n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.imshow(image_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bbox_images = []\n",
    "for image_path in test_images:\n",
    "    image = PImage.open(os.path.join('blind', image_path))\n",
    "    # the array based representation of the image will be used later in order to prepare the\n",
    "    # result image with boxes and labels on it.\n",
    "    (im_width, im_height) = image.size\n",
    "    image_np = np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)\n",
    "    data = image_np.tolist()\n",
    "    payload_scoring = {\n",
    "      \"input_data\": [{\n",
    "        \"values\": [data]\n",
    "      }]\n",
    "    }\n",
    "    predictions = client.deployments.score('<deployment-id>', payload_scoring)\n",
    "    for x in predictions['predictions']:\n",
    "        if x['id'] == 'detection_scores':\n",
    "            scores = x['values'][0]\n",
    "        if x['id'] == 'detection_boxes':\n",
    "            boxes = x['values'][0]\n",
    "        if x['id'] == 'num_detections':\n",
    "            num = x['values'][0]\n",
    "        if x['id'] == 'detection_classes':\n",
    "            classes = x['values'][0]\n",
    "    \n",
    "    vis_util.visualize_boxes_and_labels_on_image_array(\n",
    "            image_np,\n",
    "            np.squeeze(boxes),\n",
    "            np.squeeze(classes).astype(np.int32),\n",
    "            np.squeeze(scores),\n",
    "            category_index,\n",
    "            use_normalized_coordinates=True,\n",
    "            line_thickness=8,\n",
    "            min_score_thresh=0.6)\n",
    "    \n",
    "    bbox_images.append(image_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "fig = plt.figure(figsize=(120, 80))  # width, height in inches\n",
    "\n",
    "for i,bbox_image in enumerate(bbox_images):\n",
    "    sub = fig.add_subplot(len(bbox_images)+1, 1, i + 1)\n",
    "    sub.imshow(bbox_image, interpolation='nearest')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tip: You can change the variable min_score_thresh if your confidence scores are too low."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}